{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "import sys\n",
    "sys.path.append('../')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO: Pandarallel will run on 16 workers.\n",
      "INFO: Pandarallel will use Memory file system to transfer data between the main process and workers.\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from numpy.lib.stride_tricks import as_strided\n",
    "from pandarallel import pandarallel\n",
    "import numpy as np\n",
    "pandarallel.initialize(progress_bar=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preprocessing (Run only 1 time)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fill_flag(sample):\n",
    "    if not isinstance(sample['Flag'], str):\n",
    "        col = 'Data' + str(sample['DLC'])\n",
    "        sample['Flag'], sample[col] = sample[col], sample['Flag']\n",
    "    return sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sliding_window(data, win=29, s=1):\n",
    "    itemsize  = data.itemsize\n",
    "    N = len(data)\n",
    "    sliding_data = as_strided(data, shape=((N - win) // s + 1, win), strides=(itemsize*s, itemsize))\n",
    "    return sliding_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "attacks = ['DoS', 'Fuzzy', 'gear', 'RPM']\n",
    "attributes = ['Timestamp', 'canID', 'DLC', \n",
    "                           'Data0', 'Data1', 'Data2', \n",
    "                           'Data3', 'Data4', 'Data5', \n",
    "                           'Data6', 'Data7', 'Flag']\n",
    "def preprocessing(attack_id, window_size, strided):\n",
    "    filename = f'../../Data/Car-Hacking/{attacks[attack_id]}_dataset.csv'\n",
    "    df = pd.read_csv(filename, header=None, names=attributes)\n",
    "    df = df.parallel_apply(fill_flag, axis=1)\n",
    "    num_data_bytes = 8\n",
    "    for x in range(num_data_bytes):\n",
    "        df['Data'+str(x)] = df['Data'+str(x)].map(lambda x: int(x, 16), na_action='ignore')\n",
    "    df = df.fillna(0)\n",
    "    data_cols = ['Data{}'.format(x) for x in range(num_data_bytes)]\n",
    "    df[data_cols] = df[data_cols].astype(int) \n",
    "    df['Data'] = df[data_cols].values.tolist()\n",
    "    df['Flag'] = (df['Flag'] == 'T')\n",
    "    sliding_label = sliding_window(df['Flag'].to_numpy(), win=window_size, s=strided)\n",
    "    sliding_data = sliding_window(df['Data'].to_numpy(), win=window_size, s=strided)\n",
    "    labels = np.any(sliding_label, axis=1).astype('int8')\n",
    "    pp_df = pd.DataFrame({\n",
    "    'data_seq': pd.Series(sliding_data.tolist()),\n",
    "    'label': pd.Series(sliding_label.tolist())\n",
    "    }, index=range(len(sliding_label)))\n",
    "    pp_df['data_histogram'] = pp_df['data_seq'].parallel_apply(lambda x: np.histogram(np.array(x), bins=256)[0])\n",
    "    pp_df['label'] = pp_df['label'].parallel_apply(lambda x: (attack_id + 1) if any(x) else 0)\n",
    "    return pp_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LABEL: DoS\n",
      "LABEL: Fuzzy\n",
      "LABEL: gear\n",
      "LABEL: RPM\n"
     ]
    }
   ],
   "source": [
    "df_list = []\n",
    "for i, a in enumerate(attacks): \n",
    "    print(f'LABEL: {a}')\n",
    "    df = preprocessing(i, window_size=30, strided=10)\n",
    "    df_list.append(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.concat(df_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>data_seq</th>\n",
       "      <th>label</th>\n",
       "      <th>data_histogram</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>[[5, 33, 104, 9, 33, 33, 0, 111], [254, 91, 0,...</td>\n",
       "      <td>0</td>\n",
       "      <td>[108, 0, 2, 2, 1, 4, 0, 1, 7, 2, 0, 3, 1, 0, 0...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>[[229, 127, 0, 0, 72, 127, 11, 172], [0, 0, 0,...</td>\n",
       "      <td>0</td>\n",
       "      <td>[110, 1, 2, 1, 1, 3, 0, 1, 8, 1, 1, 2, 3, 0, 1...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>[[64, 187, 127, 20, 17, 32, 0, 20], [0, 0, 0, ...</td>\n",
       "      <td>0</td>\n",
       "      <td>[114, 1, 1, 2, 0, 3, 0, 0, 4, 1, 1, 1, 4, 3, 2...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>[[11, 128, 0, 255, 69, 128, 12, 133], [14, 128...</td>\n",
       "      <td>0</td>\n",
       "      <td>[96, 1, 3, 1, 0, 3, 0, 0, 6, 3, 1, 1, 3, 3, 2,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>[[5, 33, 104, 9, 33, 33, 0, 111], [64, 187, 12...</td>\n",
       "      <td>0</td>\n",
       "      <td>[109, 0, 2, 2, 0, 4, 0, 0, 4, 3, 0, 0, 1, 3, 3...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>462163</th>\n",
       "      <td>[[0, 32, 0, 0, 0, 0, 0, 0], [0, 64, 96, 255, 1...</td>\n",
       "      <td>0</td>\n",
       "      <td>[115, 3, 1, 0, 0, 3, 3, 3, 3, 4, 1, 0, 0, 0, 0...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>462164</th>\n",
       "      <td>[[0, 128, 0, 0, 48, 127, 6, 68], [0, 0, 0, 0, ...</td>\n",
       "      <td>0</td>\n",
       "      <td>[107, 2, 2, 0, 0, 3, 1, 3, 2, 4, 2, 0, 0, 0, 0...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>462165</th>\n",
       "      <td>[[0, 64, 96, 255, 126, 133, 9, 0], [255, 0, 0,...</td>\n",
       "      <td>0</td>\n",
       "      <td>[108, 3, 1, 0, 0, 4, 0, 3, 5, 4, 2, 0, 0, 0, 1...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>462166</th>\n",
       "      <td>[[5, 34, 28, 10, 34, 30, 0, 111], [254, 89, 0,...</td>\n",
       "      <td>0</td>\n",
       "      <td>[110, 2, 2, 0, 2, 3, 0, 0, 6, 5, 2, 0, 0, 0, 1...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>462167</th>\n",
       "      <td>[[0, 0, 0, 0, 0, 0, 0, 0], [41, 39, 39, 35, 0,...</td>\n",
       "      <td>0</td>\n",
       "      <td>[101, 2, 2, 0, 2, 4, 0, 0, 6, 5, 2, 0, 0, 1, 1...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1656939 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 data_seq  label  \\\n",
       "0       [[5, 33, 104, 9, 33, 33, 0, 111], [254, 91, 0,...      0   \n",
       "1       [[229, 127, 0, 0, 72, 127, 11, 172], [0, 0, 0,...      0   \n",
       "2       [[64, 187, 127, 20, 17, 32, 0, 20], [0, 0, 0, ...      0   \n",
       "3       [[11, 128, 0, 255, 69, 128, 12, 133], [14, 128...      0   \n",
       "4       [[5, 33, 104, 9, 33, 33, 0, 111], [64, 187, 12...      0   \n",
       "...                                                   ...    ...   \n",
       "462163  [[0, 32, 0, 0, 0, 0, 0, 0], [0, 64, 96, 255, 1...      0   \n",
       "462164  [[0, 128, 0, 0, 48, 127, 6, 68], [0, 0, 0, 0, ...      0   \n",
       "462165  [[0, 64, 96, 255, 126, 133, 9, 0], [255, 0, 0,...      0   \n",
       "462166  [[5, 34, 28, 10, 34, 30, 0, 111], [254, 89, 0,...      0   \n",
       "462167  [[0, 0, 0, 0, 0, 0, 0, 0], [41, 39, 39, 35, 0,...      0   \n",
       "\n",
       "                                           data_histogram  \n",
       "0       [108, 0, 2, 2, 1, 4, 0, 1, 7, 2, 0, 3, 1, 0, 0...  \n",
       "1       [110, 1, 2, 1, 1, 3, 0, 1, 8, 1, 1, 2, 3, 0, 1...  \n",
       "2       [114, 1, 1, 2, 0, 3, 0, 0, 4, 1, 1, 1, 4, 3, 2...  \n",
       "3       [96, 1, 3, 1, 0, 3, 0, 0, 6, 3, 1, 1, 3, 3, 2,...  \n",
       "4       [109, 0, 2, 2, 0, 4, 0, 0, 4, 3, 0, 0, 1, 3, 3...  \n",
       "...                                                   ...  \n",
       "462163  [115, 3, 1, 0, 0, 3, 3, 3, 3, 4, 1, 0, 0, 0, 0...  \n",
       "462164  [107, 2, 2, 0, 0, 3, 1, 3, 2, 4, 2, 0, 0, 0, 0...  \n",
       "462165  [108, 3, 1, 0, 0, 4, 0, 3, 5, 4, 2, 0, 0, 0, 1...  \n",
       "462166  [110, 2, 2, 0, 2, 3, 0, 0, 6, 5, 2, 0, 0, 0, 1...  \n",
       "462167  [101, 2, 2, 0, 2, 4, 0, 0, 6, 5, 2, 0, 0, 1, 1...  \n",
       "\n",
       "[1656939 rows x 3 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_df = pd.DataFrame([pd.Series(x) for x in df['data_histogram']])\n",
    "new_df.columns = ['{}'.format(x+1) for x in new_df.columns]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>...</th>\n",
       "      <th>247</th>\n",
       "      <th>248</th>\n",
       "      <th>249</th>\n",
       "      <th>250</th>\n",
       "      <th>251</th>\n",
       "      <th>252</th>\n",
       "      <th>253</th>\n",
       "      <th>254</th>\n",
       "      <th>255</th>\n",
       "      <th>256</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>108</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>110</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>114</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>96</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>109</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1656934</th>\n",
       "      <td>115</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1656935</th>\n",
       "      <td>107</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1656936</th>\n",
       "      <td>108</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1656937</th>\n",
       "      <td>110</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>5</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1656938</th>\n",
       "      <td>101</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>5</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1656939 rows × 256 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           1  2  3  4  5  6  7  8  9  10  ...  247  248  249  250  251  252  \\\n",
       "0        108  0  2  2  1  4  0  1  7   2  ...    0    0    0    0    0    0   \n",
       "1        110  1  2  1  1  3  0  1  8   1  ...    0    0    0    0    0    0   \n",
       "2        114  1  1  2  0  3  0  0  4   1  ...    0    0    0    0    0    0   \n",
       "3         96  1  3  1  0  3  0  0  6   3  ...    0    0    0    0    0    0   \n",
       "4        109  0  2  2  0  4  0  0  4   3  ...    0    0    0    0    0    0   \n",
       "...      ... .. .. .. .. .. .. .. ..  ..  ...  ...  ...  ...  ...  ...  ...   \n",
       "1656934  115  3  1  0  0  3  3  3  3   4  ...    0    0    0    0    0    0   \n",
       "1656935  107  2  2  0  0  3  1  3  2   4  ...    0    0    0    0    0    0   \n",
       "1656936  108  3  1  0  0  4  0  3  5   4  ...    0    1    0    0    0    0   \n",
       "1656937  110  2  2  0  2  3  0  0  6   5  ...    0    1    0    0    0    0   \n",
       "1656938  101  2  2  0  2  4  0  0  6   5  ...    0    2    1    0    0    0   \n",
       "\n",
       "         253  254  255  256  \n",
       "0          0    0    2    8  \n",
       "1          0    0    2    9  \n",
       "2          0    0    1    9  \n",
       "3          0    0    2   10  \n",
       "4          0    0    1    9  \n",
       "...      ...  ...  ...  ...  \n",
       "1656934    0    0    1   12  \n",
       "1656935    0    0    2    9  \n",
       "1656936    0    0    1   12  \n",
       "1656937    0    0    2    9  \n",
       "1656938    0    0    2    9  \n",
       "\n",
       "[1656939 rows x 256 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = new_df.to_numpy()\n",
    "y = df['label']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.savez_compressed('../../Data/Car-Hacking/full_histogram.npz', X=X, y=y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Modeling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = np.load('../../Data/Car-Hacking/full_histogram.npz')\n",
    "X,  y = data['X'], data['y']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import faiss\n",
    "class FaissKNeighbors:\n",
    "    def __init__(self, k=5):\n",
    "        self.index = None\n",
    "        self.y = None\n",
    "        self.k = k\n",
    "\n",
    "    def fit(self, X, y):\n",
    "        self.index = faiss.IndexFlatL2(X.shape[1])\n",
    "        self.index.add(X.astype(np.float32))\n",
    "        self.y = y\n",
    "\n",
    "    def predict(self, X):\n",
    "        distances, indices = self.index.search(X.astype(np.float32), k=self.k)\n",
    "        votes = self.y[indices]\n",
    "        predictions = np.array([np.argmax(np.bincount(x)) for x in votes])\n",
    "        return predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.model_selection import StratifiedShuffleSplit\n",
    "from utils import cal_metric"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CV:  1\n",
      "{'fnr': array([0.00879808, 0.58000368, 1.13926994, 0.22710468, 0.16425121]), 'rec': array([0.99991202, 0.99419996, 0.9886073 , 0.99772895, 0.99835749]), 'pre': array([0.99720199, 0.99935221, 0.99984324, 1.        , 1.        ]), 'f1': array([0.99855517, 0.99676943, 0.99419352, 0.99886319, 0.99917807])}\n",
      "CV:  2\n",
      "{'fnr': array([0.00488782, 0.52476524, 1.19352089, 0.21126017, 0.16908213]), 'rec': array([0.99995112, 0.99475235, 0.98806479, 0.9978874 , 0.99830918]), 'pre': array([0.99721182, 0.99990746, 0.99968635, 1.        , 1.        ]), 'f1': array([0.99857959, 0.99732324, 0.9938416 , 0.99894258, 0.99915387])}\n",
      "CV:  3\n",
      "{'fnr': array([0.00391026, 0.39587553, 1.12376967, 0.22710468, 0.13043478]), 'rec': array([0.9999609 , 0.99604124, 0.9887623 , 0.99772895, 0.99869565]), 'pre': array([0.99748413, 0.99972279, 0.99992162, 1.        , 1.        ]), 'f1': array([0.99872098, 0.99787862, 0.99431065, 0.99886319, 0.9993474 ])}\n",
      "CV:  4\n",
      "{'fnr': array([0.00488782, 0.50635242, 1.40277455, 0.23766769, 0.15458937]), 'rec': array([0.99995112, 0.99493648, 0.98597225, 0.99762332, 0.99845411]), 'pre': array([0.99694941, 0.99972248, 0.99984282, 1.        , 1.        ]), 'f1': array([0.99844801, 0.99732374, 0.99285909, 0.99881025, 0.99922646])}\n",
      "CV:  5\n",
      "{'fnr': array([0.00391026, 0.51555883, 1.11601953, 0.23238618, 0.17391304]), 'rec': array([0.9999609 , 0.99484441, 0.9888398 , 0.99767614, 0.99826087]), 'pre': array([0.99727018, 0.99972245, 0.99992163, 1.        , 1.        ]), 'f1': array([0.99861373, 0.99727747, 0.99434984, 0.99883672, 0.99912968])}\n"
     ]
    }
   ],
   "source": [
    "sss = StratifiedShuffleSplit(n_splits=5, random_state=0)\n",
    "ys = []\n",
    "y_preds = []\n",
    "total_results = {\n",
    "    'fnr': np.zeros(5),\n",
    "    'rec': np.zeros(5),\n",
    "    'pre': np.zeros(5),\n",
    "    'f1': np.zeros(5)\n",
    "}\n",
    "for i, (train_index, test_index) in enumerate(sss.split(X, y)):\n",
    "    print('CV: ', i + 1)\n",
    "    X_train, X_test = X[train_index], X[test_index]\n",
    "    y_train, y_test = y[train_index], y[test_index]\n",
    "    ys.append(y_test)\n",
    "    # Prediction\n",
    "    fast_knn = FaissKNeighbors(k=5)\n",
    "    fast_knn.fit(X_train, y_train)\n",
    "    y_pred = fast_knn.predict(X_test)\n",
    "    y_preds.append(y_pred)\n",
    "    ############\n",
    "    cm, results = cal_metric(y_test, y_pred)\n",
    "    print(results)\n",
    "    for k, v in total_results.items():\n",
    "        v += results[k]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'fnr': array([0.00527885, 0.50451114, 1.19507091, 0.22710468, 0.15845411]),\n",
       " 'rec': array([0.99994721, 0.99495489, 0.98804929, 0.99772895, 0.99841546]),\n",
       " 'pre': array([0.99722351, 0.99968548, 0.99984313, 1.        , 1.        ]),\n",
       " 'f1': array([0.9985835 , 0.9973145 , 0.99391094, 0.99886318, 0.9992071 ])}"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "total_results = {k: v/5 for k, v in total_results.items()}\n",
    "total_results\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.7.10 ('torch')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "ec8a7a313ab33d199c8aa698bb86bd912b8385ce4922a6e184e3f5edd5eb95f6"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
